/*
 * Dataset.h
 *
 *  Created on: Jan 15, 2011
 *      Author: LyonsDesktop
 */

#ifndef DATASET_H_
#define DATASET_H_

#include "Formats/CSV.h"
#include <vector>
#include <iostream>
#include <string>
#include <sstream>
#include <cfloat>
#include <set>

namespace CLEVER
{
namespace Datasets
{



//! An attribute in a dataset
class Attribute
{
public:
	//! The attribute's name
	std::string Name;
	//! The attribute's key set
	std::vector<std::string> Keys;

	void ClearKeys();

	void RemoveKey(std::string& rkey);
	void RemoveKeys(std::vector<std::string>& rkeys);

	void AddKey(std::string& rkey);
	void AddKeys(std::vector<std::string>& rkeys);

	//! Returns the key in the key set at the zero-based index, or an empty string if the index is greater than or equal to the number of keys in the key set.
	std::string GetKey(int index);
	//! Get the index of the specified key, or -1 if the key does not exist within this attribute's key set.
	int GetIndex(std::string& key);

	//! Minimum value
	double dmin;
	//! Maximum value
	double dmax;

	//Attribute(): dmin(DBL_MIN), dmax(DBL_MAX){ }
	Attribute(): dmin(DBL_MIN), dmax(DBL_MAX){}
};

//! A class that stores elements of the templated type V with attributes, real data and meta data.
template<typename V>
class DataStorage
{
public:
	std::vector<Attribute> attributes;
	std::vector< std::vector<V> > data;

	//! Returns the column of data
	std::vector<V> GetDataColumn(int iAtt)
	{
		std::vector<V> rd;
		rd.reserve(data.size());
		for (size_t i = 0; i < data.size(); i++)
		{
			rd.push_back(data[i][iAtt]);
		}
		return rd;
	}
	//! Appends rows to the end of the stored data
	/*!
					\param num The number of rows to append.
	 */
	virtual int Append(int num = 1)
	{ //PROFILE_FUNC
		std::vector<V> metarow;
		for (size_t i = 0; i < attributes.size(); i++) { metarow.push_back(0); }
		int next = data.size();
		for (int i = 0; i < num; i++) { data.push_back(metarow); }
		return next;
	}

	virtual Attribute GetAttribute(int iAtt)
	{
		return attributes[iAtt];
	}

	virtual void SetAttribute(int iAtt, Attribute a)
	{
		attributes[iAtt] = a;
	}

	__inline__  virtual V GetValue(int rExample, int iAtt)
	{
		return data[rExample][iAtt];
	}

	__inline__  virtual void SetValue(int rExample, int iAtt, V& value)
	{
		data[rExample][iAtt] = value;
	}

	virtual void Clear()
	{
		attributes.clear();
		data.clear();
	}
};
class DoubleArrayStorage : public DataStorage<double>
{
public:
	std::vector<Attribute> attributes;
	double* data;
	unsigned _nExample, _nAtt;

	DoubleArrayStorage(){};

	DoubleArrayStorage(const DoubleArrayStorage& p)
	{
		attributes=p.attributes;
		data=new double[p._nExample*p._nAtt];
		for(size_t i=0;i<p._nExample*p._nAtt;i++)
		{
				data[i]=p.data[i];
		}
		_nExample=p._nExample;
		_nAtt=p._nAtt;
	}

//	~DoubleArrayStorage()
//	{
//		delete[] data;
//	}
	//! Returns the column of data
	double* GetDataColumn(int iAtt)
	{
		double* rd=new double[_nExample];

		for (unsigned i = 0L; i < _nExample; i++)
		{
			rd[i]=data[getIndex(i,iAtt)];
		}
		return rd;
	}

	virtual void reserve(unsigned rExamples,  int rAtt)
	{
		data=new double[rExamples*rAtt];
		_nExample=rExamples;
		_nAtt=rAtt;
	}

	//! Appends rows to the end of the stored data
	/*!
					\param num The number of rows to append.
	 */
	virtual int Append(int num = 1) //This should not be called, double array dataset cannot append new rows
	{ //PROFILE_FUNC

		return 0;
	}

	virtual Attribute GetAttribute(int iAtt)
	{
		return attributes[iAtt];
	}

	virtual void SetAttribute(int iAtt, Attribute a)
	{
		attributes[iAtt] = a;
	}

	__inline__  virtual double GetValue(int rExample, int iAtt)
	{
	    double data0 = data[getIndex(rExample, iAtt)];
	    return data0;
	}

	__inline__  virtual void SetValue(int rExample, int iAtt, double& value)
	{
	    int index = getIndex(rExample, iAtt);
	    data[index] = value;
	}

	virtual void Clear()
	{
		attributes.clear();
		delete data;
	}

	long long getIndex(unsigned rExample, int rAtt)
	{
	    int rExample0 = (long long )((rExample * _nAtt + rAtt));
	    return rExample0;
	}
};


template<typename T> class Dataset;
//! Represents a specification for data in a dataset, either in memory or from a file.
  class Dataspec
  {
  public:
          //! Load the dataspec from file.
          /*!
          \param srcfile An XML file.
          */
          Dataspec(const char* srcfile);
          std::vector<CLEVER::Datasets::Attribute> RealAttributes;
          std::vector<CLEVER::Datasets::Attribute> MetaAttributes;

          //! Load the dataspec from an existing dataset.
          /*
          \param dataset An existing dataset.
          */
          //template <typename T>
          //Dataspec( Dataset<T>* dataset);
          template<typename T>
          Dataspec(Dataset<T>* dataset)
          {
            for (int i = 0; i < dataset->RealData.attributes.size(); i++)
              {
                RealAttributes.push_back(dataset->RealData.attributes[i]);
              }
            for (int i = 0; i < dataset->MetaData.attributes.size(); i++)
              {
                MetaAttributes.push_back(dataset->MetaData.attributes[i]);
              }
          }
  };

//! Encapsulates the I/O behaviour of a set of data.
template<typename T>
class Dataset
{
public:
	DataStorage<T> MetaData;
	DataStorage<T> RealData;

	~Dataset(){
		MetaData.Clear();
		RealData.Clear();
	}
	virtual int Size() { return (int)RealData.data.size(); }

	//! Returns a pointer of type T to the real data of the specified example in the dataset.
	__inline__  virtual T* GetValuePtr(int iExample) { return &RealData.data[iExample][0]; }

	__inline__  virtual T GetValue(int iExample, int iAtt) { return RealData.GetValue(iExample, iAtt); }
	__inline__  virtual void SetValue(int iExample, int iAtt, T value) { RealData.SetValue(iExample, iAtt, value); }

	__inline__  virtual T GetMeta(int iExample, int iAtt) { return MetaData.GetValue(iExample, iAtt); }
	__inline__  virtual void SetMeta(int iExample, int iAtt, T value) { MetaData.SetValue(iExample, iAtt, value); }

	//! Clone the specified example from the source dataset into the destination
	/*!
					\param source The dataset to clone from.
					\param iSource The index of the example in the source dataset.
					\param dest The destination dataset, where the cloned example will be appended.
	 */
	int static CloneExample(Dataset<T>* source, int iSource, Dataset<T>* dest)
							{
		int idest = dest->Append();
		for (int iAtt = 0; iAtt < source->GetNumAttributes(); iAtt++)
		{
			dest->SetValue(idest, iAtt, source->GetValue(iSource, iAtt));
		}
		for (int iAtt = 0; iAtt < source->GetNumMetaAttributes(); iAtt++)
		{
			dest->SetMeta(idest, iAtt, source->GetMeta(iSource, iAtt));
		}
		return idest;
							}

	//! Clones the examples whose indices are given into the target dataset
	virtual void Subset(Dataset<T>* target, std::vector<int> indices)
	{
		target->MetaData.attributes = MetaData.attributes;
		target->RealData.attributes = RealData.attributes;
		for (unsigned int i = 0; i < indices.size(); i++)
		{
			CloneExample(this, indices[i], target);
		}
	}

	//! Splits the dataset into an array of datasets based on matching values in the specifed meta.
	virtual std::vector<Dataset<T> > Split(int metaindex)
	{ //PROFILE_FUNC
	  std::vector<Dataset<T> > d;
	  d.reserve(MetaData.attributes[metaindex].Keys.size());
	  for (unsigned int i = 0; i < MetaData.attributes[metaindex].Keys.size(); i++)
	    {
	      Dataset<T> D;
	      D.MetaData.attributes = MetaData.attributes;
	      D.RealData.attributes = RealData.attributes;
	      D.MetaData.data.reserve(MetaData.data.size());
	      D.RealData.data.reserve(Size());

	      for (unsigned int j = 0; j < MetaData.data.size(); j++)
	        {
	          if ((size_t)GetMeta(j, metaindex) == i)
	            {
	              int dest = D.Append();
	              for (unsigned int x = 0; x < RealData.attributes.size(); x++)
	                {
	                  D.SetValue(dest, x, GetValue(j, x));
	                }
	              for (unsigned int x = 0; x < MetaData.attributes.size(); x++)
	                {
	                  D.SetMeta(dest, x, GetMeta(j, x));
	                }
	            }
	        }
	      d.push_back(D);
	    }
	  return d;
	}

	virtual int RealIndexOf(Attribute att)
	{ //PROFILE_FUNC
		for (size_t i = 0; i < RealData.attributes.size(); i++)
		{
			if (RealData.attributes[i].Name == att.Name && RealData.attributes[i].Keys == att.Keys)
			{ return i; }
		}
		return -1;
	}

	virtual int MetaIndexOf(Attribute att)
	{ //PROFILE_FUNC
		for (size_t i = 0; i < MetaData.attributes.size(); i++)
		{
			if (MetaData.attributes[i].Name == att.Name && MetaData.attributes[i].Keys == att.Keys)
			{ return i; }
		}
		return -1;
	}

	__inline__  virtual int GetNumAttributes() { return RealData.attributes.size(); }
	__inline__  virtual Attribute GetAttribute(int i) { return RealData.attributes[i]; }

	__inline__  virtual int GetNumMetaAttributes() { return MetaData.attributes.size(); };
	__inline__  virtual Attribute GetMetaAttribute(int i) { return MetaData.attributes[i]; };

	Dataset<T>* Allocate();

	//! Appends rows to the end of the stored data
	/*!
					\param num The number of rows to append.
	 */
	virtual int Append(int num = 1)
	{ //PROFILE_FUNC
		int next = Size();
		RealData.Append(num); MetaData.Append(num);
		return next;
	}

	//! Writes this dataset to a CSV file.

	virtual void WriteToCSV(const char* filename)
	{ //PROFILE_FUNC
		Datasets::Formats::CSVWriter csvwriter(filename);
		for (int i = 0; i < Size(); i++)
		{
			for (unsigned int j = 0; j < RealData.attributes.size(); j++)
			{
				csvwriter.WriteValue(RealData.data[i][j]);
			}
			for (unsigned int j = 0; j < MetaData.attributes.size(); j++)
			{
				csvwriter.WriteValue(MetaData.data[i][j]);
			}
			csvwriter.Newline();
		}
		csvwriter.Close();
	}

	//! Determine key sets for meta-data if none were loaded from the data specification.
	virtual void SetDefaultMetaKeys()
	{ //PROFILE_FUNC
		for (unsigned int _att = 0; _att < MetaData.attributes.size(); _att++)
		{
			if (MetaData.attributes[_att].Keys.empty())
			{
				Attribute att = GetMetaAttribute(_att);

				for (unsigned int i = 0; i < MetaData.data.size(); i++)
				{

					std::ostringstream buf;
//vc++ 	only
//					char buf[256];
//					sprintf_s(buf, 256, "R%d", (int)GetMeta(i, _att));

//Modified for linux

					buf << "R"<< (int)GetMeta(i, _att);
					std::string _str= buf.str();
					if (att.GetIndex(_str) == -1)
					{
						att.AddKey(_str);
					}
				}

				MetaData.attributes[_att] = att;
			}
		}
	}


	//! Loads this dataset from a CSV file with the given data specification
	/*!
					\param filename CSV input file
					\param dataspec Data specification
	 */

	virtual void LoadFromCSV(const char* filename, CLEVER::Datasets::Dataspec* dataspec)
	{ //PROFILE_FUNC
		MetaData.Clear();
		RealData.Clear();

		MetaData.attributes = dataspec->MetaAttributes;
		RealData.attributes = dataspec->RealAttributes;

		Formats::CSVReader csvreader(filename);
		for (int iline = 0; iline < csvreader.NumLines(); ++iline)
		{
			std::vector<std::string> line = csvreader.GetLine(iline);
			int lptr = 0;
			int iExample = Append();
			for (unsigned int i = 0; i < dataspec->RealAttributes.size(); i++)
			{
				std::stringstream ss(line[lptr++]);
				T v;
				ss >> v;
				SetValue(iExample, i, v);
			}
			for (unsigned int i = 0; i < dataspec->MetaAttributes.size(); i++)
			{
				std::stringstream ss(line[lptr++]);
				T v;
				ss >> v;
				SetMeta(iExample, i, v);
			}
		}

		//SetDefaultMetaKeys();
	}
};
typedef Dataset<double> NumericDataset;

class DoubleArrayDataset : public Dataset<double>
{
public:
	DoubleArrayStorage MetaData;
	DoubleArrayStorage RealData;


	~DoubleArrayDataset(){
		MetaData.Clear();
		RealData.Clear();
	}
	virtual int Size() { return (int)RealData._nExample; }

	//! Returns a pointer of type T to the real data of the specified example in the dataset.
	__inline__  virtual double* GetValuePtr(int iExample) { return &RealData.data[RealData.getIndex(iExample,0)]; }

	__inline__  virtual double GetValue(int iExample, int iAtt) { return RealData.GetValue(iExample, iAtt); }
	__inline__  virtual void SetValue(int iExample, int iAtt, double value) { RealData.SetValue(iExample, iAtt, value); }

	__inline__  virtual double GetMeta(int iExample, int iAtt) { return MetaData.GetValue(iExample, iAtt); }
	__inline__  virtual void SetMeta(int iExample, int iAtt, double value) { MetaData.SetValue(iExample, iAtt, value); }

	//! Clone the specified example from the source dataset into the destination
	/*!
					\param source The dataset to clone from.
					\param iSource The index of the example in the source dataset.
					\param dest The destination dataset, where the cloned example will be appended.
	 */
	int static CloneExample(Dataset<double>* source, int iSource, Dataset<double>* dest)// Note that the dest cannot be a doubleArrayDataset since Append() is not supported
	{
		int idest = dest->Append();
		for (int iAtt = 0; iAtt < source->GetNumAttributes(); iAtt++)
		{
			dest->SetValue(idest, iAtt, source->GetValue(iSource, iAtt));
		}
		for (int iAtt = 0; iAtt < source->GetNumMetaAttributes(); iAtt++)
		{
			dest->SetMeta(idest, iAtt, source->GetMeta(iSource, iAtt));
		}
		return idest;
	}

	//! Clones the examples whose indices are given into the target dataset, target cannot be a doubleArrayDataset since or the clone will fail
	virtual void Subset(Dataset<double>* target, std::vector<int> indices)
	{
		target->MetaData.attributes = MetaData.attributes;
		target->RealData.attributes = RealData.attributes;
		for (unsigned int i = 0; i < indices.size(); i++)
		{
			CloneExample(this, indices[i], target);
		}
	}

	//! Splits the dataset into an array of datasets based on matching values in the specifed meta(assume to be cluster ID??).
	//	The returned vector contains normal dataset, not the doubleArrayDataset
	virtual std::vector<Dataset<double> > Split(int metaindex)
	{ //PROFILE_FUNC
	  std::vector<Dataset<double> > d;
	  d.reserve(MetaData.attributes[metaindex].Keys.size());
	  for (unsigned int i = 0; i < MetaData.attributes[metaindex].Keys.size(); i++)
	    {
	      Dataset<double> D;
	      D.MetaData.attributes = MetaData.attributes;
	      D.RealData.attributes = RealData.attributes;
	      D.MetaData.data.reserve(MetaData._nExample);
	      D.RealData.data.reserve(Size());

	      for (unsigned long j = 0; j < MetaData._nExample; j++)
	        {
	          if ((size_t)GetMeta(j, metaindex) == i)
	            {
	              int dest = D.Append();
	              for (unsigned int x = 0; x < RealData.attributes.size(); x++)
	                {
	                  D.SetValue(dest, x, GetValue(j, x));
	                }
	              for (unsigned int x = 0; x < MetaData.attributes.size(); x++)
	                {
	                  D.SetMeta(dest, x, GetMeta(j, x));
	                }
	            }
	        }
	      d.push_back(D);
	    }
	  return d;
	}

	virtual int RealIndexOf(Attribute att)
	{ //PROFILE_FUNC
		for (size_t i = 0; i < RealData.attributes.size(); i++)
		{
			if (RealData.attributes[i].Name == att.Name && RealData.attributes[i].Keys == att.Keys)
			{ return i; }
		}
		return -1;
	}

	virtual int MetaIndexOf(Attribute att)
	{ //PROFILE_FUNC
		for (size_t i = 0; i < MetaData.attributes.size(); i++)
		{
			if (MetaData.attributes[i].Name == att.Name && MetaData.attributes[i].Keys == att.Keys)
			{ return i; }
		}
		return -1;
	}

	__inline__  virtual int GetNumAttributes() { return RealData.attributes.size(); }
	__inline__  virtual Attribute GetAttribute(int i) { return RealData.attributes[i]; }

	__inline__  virtual int GetNumMetaAttributes() { return MetaData.attributes.size(); };
	__inline__  virtual Attribute GetMetaAttribute(int i) { return MetaData.attributes[i]; };

	Dataset<double>* Allocate();

	//! Appends rows to the end of the stored data
	/*!
					\param num The number of rows to append.
	 */
	int Append(int num = 1)
	{ //PROFILE_FUNC
		std::cerr<<"DoubleArrayDataset cannot append new rows"<<std::endl;
		return (10/0); //Runtime Error if this is called
	}

	//! Writes this dataset to a CSV file.

	void WriteToCSV(const char* filename)
	{ //PROFILE_FUNC
		Datasets::Formats::CSVWriter csvwriter(filename);
		for (int i = 0; i < Size(); i++)
		{
			for (unsigned int j = 0; j < RealData.attributes.size(); j++)
			{
				csvwriter.WriteValue(RealData.GetValue(i,j));
			}
			for (unsigned int j = 0; j < MetaData.attributes.size(); j++)
			{
				csvwriter.WriteValue(MetaData.GetValue(i,j));
			}
			csvwriter.Newline();
//			std::cout<< "Write Record :"<<i<<"cluster ID: "<<MetaData.GetValue(i,1)<<std::endl;
		}
		csvwriter.Close();
	}

	//! Determine key sets for meta-data if none were loaded from the data specification.
	void SetDefaultMetaKeys()
	{ //PROFILE_FUNC
		for (unsigned int _att = 0; _att < MetaData.attributes.size(); _att++)
		{
			std::cout<<"SetDefaultMetaKeys, _att="<<_att<<"/"<<MetaData.attributes.size()<<std::endl;
			if (MetaData.attributes[_att].Keys.empty())
			{
				Attribute att = GetMetaAttribute(_att);

				//#pragma omp parallel for
				for (unsigned int i = 0; i < MetaData._nExample; i++)
				{

					std::ostringstream buf;
//vc++ 	only
//					char buf[256];
//					sprintf_s(buf, 256, "R%d", (int)GetMeta(i, _att));

//Modified for linux

					buf << "R"<< (int)GetMeta(i, _att);
					std::string _str= buf.str();
					if (att.GetIndex(_str) == -1)
					{
						att.AddKey(_str);
					}
					if(i%100000==0)
						std::cout<<"SetDefaultMetaKeys, _nExamples="<<i<<"/"<<MetaData._nExample<<std::endl;
				}

				MetaData.attributes[_att] = att;
			}
			std::cout<<"SetDefaultMetaKeys, _att="<<_att<<"/"<<MetaData.attributes.size()<<std::endl;
		}
	}


	//! Loads this dataset from a CSV file with the given data specification
	/*!
					\param filename CSV input file
					\param dataspec Data specification
	 */

	void LoadFromCSV(const char* filename, CLEVER::Datasets::Dataspec* dataspec, unsigned rExamples)
	{ //PROFILE_FUNC
		MetaData.attributes.clear();
		RealData.attributes.clear();

		MetaData.attributes = dataspec->MetaAttributes;
		RealData.attributes = dataspec->RealAttributes;

		int tNumMetaAtt=dataspec->MetaAttributes.size();
		int tNumRealAtt=dataspec->RealAttributes.size();
		std::set<double> clusterIDs;

		//Formats::CSVReader csvreader(filename);

		//unsigned tNumExamples=csvreader.NumLines(); //50000,24plusDays:22875132, 20PlusDays:40856951, 14PlusDays:81312391

		unsigned tNumExamples=rExamples;
		MetaData.reserve(tNumExamples,tNumMetaAtt);
		RealData.reserve(tNumExamples,tNumRealAtt);
		Formats::CSVReader csvreader(filename,RealData.data,MetaData.data,tNumExamples,tNumRealAtt,tNumMetaAtt,clusterIDs);

		CLEVER::Datasets::Attribute clusterIDAtt, dummyAtt;

//		for(std::set<double>::const_iterator p=clusterIDs.begin(); p!=clusterIDs.end();p++)
//		{
//			std::ostringstream ss;
//			ss<<"R"<<*p;
//			std::string akey=ss.str();
//			clusterIDAtt.AddKey(akey);
//		}

		std::string aString= "bcookie";
		dummyAtt.AddKey(aString);

		//set the meta attributes
		MetaData.attributes[0]=dummyAtt;
		//MetaData.attributes[1]=clusterIDAtt;


//		int iExample = 0;
//		std::vector<std::string> line;
//		for (int iline = 0; iline < csvreader.NumLines(); ++iline)
//		{
//			line = csvreader.GetLine(iline);
//			int lptr = 0;
//
//			for (unsigned int i = 0; i < dataspec->RealAttributes.size(); i++)
//			{
//				std::stringstream ss(line[lptr++]);
//				double v;
//				ss >> v;
//				SetValue(iExample, i, v);
//			}
//			for (unsigned int i = 0; i < dataspec->MetaAttributes.size(); i++)
//			{
//				std::stringstream ss(line[lptr++]);
//				double v;
//				ss >> v;
//				SetMeta(iExample, i, v);
//			}
//			iExample++;
//			std::cout<<iExample<<std::endl;
//		}

		SetDefaultMetaKeys();
	}
};
//! A dataset that deals with double-precision numerical values.


//! A subset of a numerical dataset.
class SubNumericDataset
{
public:
	SubNumericDataset(NumericDataset* dataset, std::vector<int>& replist)
	: _del(dataset), _replist(replist)
	{
	}

	__inline__  double GetValue(int iExample, int iAtt)
	{ return _del->RealData.GetValue(_replist[iExample], iAtt); }

	__inline__  void SetValue(int iExample, int iAtt, double value)
	{ _del->RealData.SetValue(_replist[iExample], iAtt, value); }

	__inline__  double GetMeta(int iExample, int iAtt)
	{ return _del->MetaData.GetValue(_replist[iExample], iAtt); }

	__inline__  void SetMeta(int iExample, int iAtt, double value)
	{ _del->MetaData.SetValue(_replist[iExample], iAtt, value); }

	__inline__  int size() { return _replist.size(); }

	//! Build a NumericDataset from this subset's associated NumericDataset and indices.
	NumericDataset* AsDataset()
							{
		NumericDataset* D = new NumericDataset();
		D->MetaData.attributes = _del->MetaData.attributes;
		D->RealData.attributes = _del->RealData.attributes;
		D->MetaData.data.reserve(_del->MetaData.data.size());
		D->RealData.data.reserve(_del->Size());
		for (int i = 0; i < _del->Size(); i++)
		{
			int dest = D->Append();
			for (unsigned int x = 0; x < _del->RealData.attributes.size(); x++)
			{
				D->SetValue(dest, x, _del->GetValue(i, x));
			}
			for (unsigned int x = 0; x < _del->MetaData.attributes.size(); x++)
			{
				D->SetMeta(dest, x, _del->GetMeta(i, x));
			}
		}
		return D;
							}

private:
	NumericDataset* _del;
	std::vector<int> _replist;
};

}
}
#endif /* DATASET_H_ */
